#!/bin/bash

# take number of snaps, volid, as parms
# schedule runs from cron (pass snaps and volid/ fs1 nightly, all others monthly)
# logging
# alerting and paralysis
# @TODO Jeff Masiello Notes: See NS-440. See fssnap_prep_cron.sh lot of stuff in here that doesn;t work or never will due to changes in
# AWS elements. Deprecated elements need cleaning up
aspr_home=/opt/amisnaps
export HOME=$aspr_home
export PATH=$PATH:$EC2_HOME/bin:$JAVA_HOME
#export JAVA_HOME=/usr/lib/jvm/java-6-openjdk
#. $HOME/aws-env
#pky=$aspr_home/.aws/pk-LTCDXKLN7GWX5RYPULNJ3LLLOJD3H2P7.pem
#crt=$aspr_home/.aws/cert-LTCDXKLN7GWX5RYPULNJ3LLLOJD3H2P7.pem
rndsd=$(head -1 /dev/urandom | od -N 1 | awk '{ print $2 }')
RANDOM=$rndsd
status=0

# for validation of snapshots, id and zone in which THIS server is located:
admsrvid=$(curl -s "http://169.254.169.254/latest/meta-data/instance-id")
echo $admsrvid
adsrvzone=$(curl -s  "http://169.254.169.254/latest/meta-data/placement/availability-zone")
echo $adsrvzone
# where logfile is written:
lgdr=/var/log/amisnaps
lgfl=$lgdr/snaps.log
tdt="$(date +"%Y-%m-%d")"
ddt="$(date)"

# description describing the backup target:
dsc="amisnap created: $tdt"

cd $aspr_home
echo "BEGIN************************************************************" | tee -a $lgfl

# Looks for a file called "badfile", which is left on prior errors.
# If a previous backup is invalid, a "badfile" will be generated, and no further backups can
# be made, until an administrator checks the log, and deletes the "badfile" to re-enable backups.
# @JAM modifying to remove the old adfile. the email is sent on previous error. will send again, then
# remove badfile and continue so as to not block all backups.
if [ -e "badfile" ]; then
    echo "$ddt: Backup attempted while badfile present, review log for prior backup errors; exiting" | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup attempted while badfile present on "`/bin/hostname` < $lgfl
    fi
    rm badfile
fi

#first parm 
# number of previous snapshots to retain:
if [ -n "$1" ]; then
    snaps=$1
else
    echo "$ddt: no retention number submitted; setting badfile and exiting" | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
fi
if [ $snaps -lt 5 ]; then
    # safety. . .
    snaps=$((5))
fi
#second parm 
#volume identifier 
if [ -n "$2" ]; then
    vid=$2
else
    # no volid submitted?
    echo "$ddt: no volume id submitted; setting badfile and exiting" | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
    #vid="vol-170cb279"
fi
#third parm 
# volume size
if [ -n "$3" ]; then
    volsize=$3
else
    #no volume size submitted
    echo "$ddt: no volume submitted submitted; setting badfile and exiting" | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
fi

#fourth parm 
# device specifier
if [ -n "$4" ]; then
    devdesig=$4
else
    devdesig="f"
fi

#fifth parm 
# optional
if [ -n "$5" ]; then
    novalid=$5
fi

#sixth parm
# host name tag (for snapshot description)
if [ -n "$6" ]; then
    hntag=$6
else
    hntag="not-specified"
fi

#sixth parm
# host name tag (for snapshot description)
if [ -n "$7" ]; then
    region=$7
else
    region=$AWS_DEFAULT_REGION
fi


function clean_up {
    # Perform program exit housekeeping
    echo "$ddt:  SIGHUP, SIGINT, SIGTERM, or SIGQUIT caught; setting badfile and exiting" | tee -a $lgfl
    if [ -s "$dbdumper_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $dbdumper_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $dbdumper_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
echo "test" > test.txt
    exit
}
trap clean_up SIGHUP SIGINT SIGTERM SIGQUIT

#rm -f snapidlst.txt
#rm -f srtsnps.txt
#rm -f prvsnps.txt
#rm -f snapout.txt
#rm -f allsnps.txt
#rm -f srtallsnps.txt
#rm -f lastsnap.txt
#rm -f cr8vol.txt
#rm -f volid.txt
#rm -f atchout.txt
#rm -f atverif.txt
#rm -f atchstatus.txt
#rm -f dtchout.txt
#rm -f voluse.txt
#rm -f atchstatus.txt
#rm -f atverif.txt
#rm -f usstatus.txt
#rm -f voluse.txt
#rm -f verifydetach.txt
#rm -f verifyattach.txt
#rm -f thisvol.txt

echo "$ddt:  Initiating backup of: $vid." | tee -a $lgfl
# when the volume exists, there will be output in thisvol.txt
# if the volume does not exist, thisvol.txt will be empty, but the
# console will say "Client.InvalidVolume.NotFound: " (etc).
# This condition will exit with code 1, so we exit the script
ec2-describe-volumes --region $region $vid > thisvol.txt 2>error.txt

status=$?
displaystatus=$status
if [ -z "$status" ]; then 
    #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""... now considered an error
    $status=1
fi

if [[ $status -gt 0 ]]; then
    if [ -s "error.txt" ]; then
        echo -n "$ddt:  " | tee -a $lgfl
        cat error.txt | tee -a $lgfl
    fi
    rm -f error.txt
    echo "$ddt:  Describe Volumes exited with '$displaystatus'. Typically, exit code 1 means Client.InvalidVolume.NotFound, so the volume specified was incorrect, or there was no volume specified, or the tool couldn't locate the volume as-specified for whatever reason.  It also can be Client.InvalidParameterValue or something else. Note that we have seen 'empty' exit codes as well. We consider this an error.  Amisnaps was unable to use the ec2 command line tools. Exiting backup and creating 'badfile'." | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
fi
rm -f error.txt

thisvol="$(wc -l thisvol.txt | awk '{print $1}')"
thisinstid="$(grep "ATTACHMENT" thisvol.txt | awk '{print $3}')"
thisdevid="$(grep "ATTACHMENT" thisvol.txt | awk '{print $4}')"

if [[ $thisvol -eq 0 ]]; then
    #we should never get here
    echo "$ddt: specified volume $vid was not found! Setting badfile and exiting." | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
fi

echo "$ddt: Describe Volumes found a volume, locating snapshots. . . " | tee -a $lgfl


ec2-describe-snapshots --region $region  | grep "$vid" > prvsnps.txt
ec2-describe-snapshots --region $region 2>error.txt
status=$?
displaystatus=$status
if [ -z "$status" ]; then
    #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
    # I want to capture > 1 situations.
    $status=1
fi

if [[ $status -gt 0 ]]; then
    if [ -s "error.txt" ]; then
        echo -n "$ddt:  " | tee -a $lgfl
        cat error.txt | tee -a $lgfl
    fi
    rm -f error.txt
    echo "$ddt:  describe snapshots exited with '$displaystatus. amisnaps was unable to use the ec2 command line tools. Exiting backup." | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
fi
rm -f error.txt

#determine number of previous snapshots.
snpout="$(wc -l prvsnps.txt)"
snpct="$(wc -l prvsnps.txt | awk '{print $1}')"

echo "$ddt: $snpct previous snapshots of this volume were found." | tee -a $lgfl

#perform the snapshot
echo "$ddt: Creating snapshot." | tee -a $lgfl
ec2-create-snapshot --region $region $vid -d "$dsc of $vid attached as $thisdevid on $thisinstid known as $hntag" > snapout.txt 2>error.txt
status=$?
displaystatus=$status
if [ -z "$status" ]; then 
    #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
    # I want to capture > 1 situations.
    $status=1
fi

if [[ $status -gt 0 ]]; then
    if [ -s "error.txt" ]; then
        echo -n "$ddt:  " | tee -a $lgfl
        cat error.txt | tee -a $lgfl
    fi
    rm -f error.txt
    echo "$ddt:  create snapshot exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
fi
rm -f error.txt

#validate snapshot
#get last snapid:
cat snapout.txt | awk '{print $2}' > lastsnap.txt

lsnpid=""
lsnpid=`cat lastsnap.txt`
# test $lsnpid for "snap"
lsnparr=(`echo $lsnpid | tr '-' ' '`)
if [ "${lsnparr[0]}" != "snap" ]; then
    # this snapshot was not valid; (create snapshot exited with 0,
    # but there was no valid "snap-" in the 2nd field of the output, so, exit script
    echo "$ddt: Snapshot $lsnpid was not a valid snapshot id. Setting badfile. Exiting." | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
fi

echo "$ddt: New Snapshot id: $lsnpid." | tee -a $lgfl



if [ "$novalid" != "novalid" ]; then
    echo "$ddt: Validating snapshot: $lsnpid. . ." | tee -a $lgfl

    # with a valid snapshot id, we can proceed with further validation
    newsnapstatus=""
    cr8snapct=`expr 0`
    while [[ "$newsnapstatus" != "completed" && $cr8snapct -le 10 ]]
    do
        #retry this command at least 10 times or until we get a "completed" status
        # . . . whichever comes first
        ec2-describe-snapshots --region $region  "$lsnpid" | awk '{print $4}' > newsnapstatus.txt
        newsnapstatus=`cat newsnapstatus.txt`
        cr8snapct=`expr $cr8snapct + 1`
        echo "$ddt: ... try ${cr8snapct}: status is: ${newsnapstatus}." | tee -a $lgfl
        sleep 10
    done
    echo "$ddt: After $cr8snapct tries, snapshot status is: $newsnapstatus." | tee -a $lgfl

    # should be "completed" by now, or, we have exceeded retries
    ec2-describe-snapshots --region $region  "$lsnpid" | tee -a $lgfl
    ec2-describe-snapshots --region $region  "$lsnpid"  2>error.txt
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        if [ -s "error.txt" ]; then
            echo -n "$ddt:  " | tee -a $lgfl
            cat error.txt | tee -a $lgfl
        fi
        rm -f error.txt
        echo "$ddt:  Describe snapshot for $lsnpid exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    rm -f error.txt
    echo "$ddt: Snapshot $lsnpid was described (found) . . . creating volume. . ." | tee -a $lgfl

    #IN SOME INSTANCES:  create volume will exit with status 1:
    #Client.Incorrect.State: snapshot 'snap-<id>' is not 'completed'.
    #Adding a sleep statement, to give Amazon some time to catch up possibly
    # during some high-latency periods?
    #JAM some snaps take a VERY long time to create so in the end this approach won't work

    sleep 30

    #create the volume
    ec2-create-volume --region $region  -s $volsize --snapshot $lsnpid -z $adsrvzone > cr8vol.txt 2>error.txt
    #VOLUME  vol-40ed5321    500     snap-eebf299f   us-east-1a      creating       2012-07-16T22:52:19+0000
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        if [ -s "error.txt" ]; then
            echo -n "$ddt:  " | tee -a $lgfl
            cat error.txt | tee -a $lgfl
        fi
        rm -f error.txt
        echo "$ddt:  Create volume exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    rm -f error.txt
    echo "$ddt: Volume was created for $lsnpid, finding volume ID. . . " | tee -a $lgfl

    #verify created volume
    #get volid:
    cat cr8vol.txt | awk '{print $2}' > volid.txt
    crdvolid=""
    crdvolid=`cat volid.txt`
    # test $crvolid for "vol"
    crdvolarr=(`echo $crdvolid | tr '-' ' '`)
    if [ "${crdvolarr[0]}" != "vol" ]; then
        # this volume is not valid; (create volume exited with 0?),
        # but there was no valid "vol-" in the 2nd field of the output, so, exit script
        echo "$ddt: Volume $crdvolid was not a valid volume id. Setting badfile. Exiting." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    echo "$ddt: Volume id for $lsnpid is $crdvolid, and is valid. Attaching volume to admin server. " | tee -a $lgfl

    #attach the volume
    ec2-attach-volume --region $region  $crdvolid --instance $admsrvid -d /dev/sd$devdesig > atchout.txt 2>error.txt
    #ATTACHMENT      vol-1cd6687d    i-59386d3f      /dev/sdf        attaching      2012-07-16T23:11:41+0000
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        if [ -s "error.txt" ]; then
            echo -n "$ddt:  " | tee -a $lgfl
            cat error.txt | tee -a $lgfl
        fi
        rm -f error.txt
        echo "$ddt:  attach volume exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    rm -f error.txt

    echo "$ddt: Verifying attachment of volume: $crdvolid " | tee -a $lgfl
    #verify volume attach
    atchstatus=false
    usstatus=false
    cntr=1
    while [[ "${atchstatus}" != "attached" && "${usstatus}" != "available" && cntr -lt 5 ]]
    do
        ec2-describe-volumes --region $region $crdvolid > verifyattach.txt
        cat verifyattach.txt | grep "ATTACHMENT" > atverif.txt
        cat verifyattach.txt | grep "VOLUME" > voluse.txt
        #VOLUME  vol-e87efa89    500     snap-44831835   us-east-1c      in-use  2012-07-17T20:00:50+0000
        #ATTACHMENT      vol-e87efa89    i-59386d3f      /dev/sdg        attached       2012-07-17T20:02:40+0000
        cat atverif.txt | awk '{print $5}' > atchstatus.txt
        cat voluse.txt | awk '{print $6}' > usstatus.txt
        atchstatus=`cat atchstatus.txt`
        usstatus=`cat usstatus.txt`
        #rm -f atchstatus.txt
        #rm -f atverif.txt
        #rm -f usstatus.txt
        #rm -f voluse.txt
        #rm -f verifyattach.txt
        # $atchstatus = "attached" if attached, "False" if not, or "attaching" if not YET.
        echo "$ddt:  describe volumes attach status returned $atchstatus for $crdvolid on try $cntr." | tee -a $lgfl
        # $usstatus = "in-use" if attached, "False" if something is wrong, "available" if not yet attached.
        echo "$ddt:  describe volumes volume status returned $usstatus for $crdvolid on try $cntr." | tee -a $lgfl
        cntr=`expr $cntr + 1`
        sleep 10
    done

    if [ "${atchstatus}" != "attached" ]; then
        #we exited the loop because $cntr exceeded 5
        # (I won't test, I will assume).
        echo "$ddt:  Describe volumes did not return attached after $cntr retries. This is an ongoing issue with Amazon, and either the previous instance of /dev/sdf did not unmount cleanly, or the created volume is bad, or there is a bug in Amazon's API. This situation can usually be corrected by rebooting this backup server. Setting badfile. Exiting." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    echo "$ddt:  describe volumes returned $atchstatus for $crdvolid on try $cntr. Mounting attached volume." | tee -a $lgfl
    sleep 10

    echo "$ddt: Mounting volume $crdvolid as /dev/sd$devdesig (will be /dev/xvd$devdesig). " | tee -a $lgfl
    #mount (check exit code)
    # (volume was mounted as "/dev/sdf" but the system creates it as "/dev/xvdf"
    mount /dev/xvd$devdesig /opt/amisnaps/test
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        echo "$ddt:  mount volume exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    echo "$ddt:  Volume mounted successfully:" | tee -a $lgfl
    mount | tee -a $lgfl
    sleep 10

    echo "$ddt: Verifying file system for $crdvolid" | tee -a $lgfl
    # check fs
    df -k --sync
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        echo "$ddt:  df exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    echo "$ddt:  df completed." | tee -a $lgfl
    df -k --sync | tee -a $lgfl
    sleep 10

    echo "$ddt: Performing checksum (md5sum) of random files on volume to verify readability on $crdvolid " | tee -a $lgfl
    #obtain list of files
    find /opt/amisnaps/test -type f > $lgdr/$tdt-$lsnpid-lst.log
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        echo "$ddt:  find exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    echo "$ddt:  find completed; output in $lgdr/$tdt-$lsnpid-lst.log." | tee -a $lgfl
    sleep 10
    
    echo "$ddt: hashing randomly selected files to verify readability. Results in $lgdr/$tdt-$lsnpid-hsh.log" | tee -a $lgfl
    
    #find # of files in list:
    qtyfiles="$(wc -l $lgdr/$tdt-$lsnpid-lst.log | awk '{print $1}')"
    echo "$ddt: Total: $qtyfiles items found on volume $crdvolid, $lsnpid." | tee -a $lgfl
    echo "$ddt: Total: $qtyfiles items found on volume $crdvolid, $lsnpid." | tee -a $lgfl/$tdt-$lsnpid-lst.log
    #select n files, randomly, from lst, and output hash values.
    failcount=$((0))
    for (( i=1; i <= 100; i++))
    do
        #echo $i
        #echo $failcount
        # take modulo of number of files from RANDOM (0-32767).
        # yeilds a range of (0-$qtyfiles)
        thisrandom=`expr $RANDOM % $qtyfiles`
        #echo $thisrandom
        #let "thisrandom %= $qtyfiles"
        awk "NR==$thisrandom" $lgdr/$tdt-$lsnpid-lst.log > tmphashtrgt.txt
        tmphashtrgt=`cat tmphashtrgt.txt`
        #echo $tmphashtrgt
        if [ -n "$tmphashtrgt" ]; then
            md5sum $tmphashtrgt | tee -a $lgfl/$tdt-$lsnpid-hsh.log
            # this could fail for permissions reasons? We're root, but these are
            # files with foreign ACLs and uids.
            if [[ $? -gt 0 ]]; then
                echo "$ddt: md5sum returned $? hashing $tmphashdrgt." | tee -a $lgfl/$tdt-$lsnpid-hsh.log
                failcount=`expr $failcount + 1`
            fi
        fi
    done

    if [[ $failcount -eq 100 ]]; then
        # every hash read failed: returned an exit code other than 0.
        # since we're picking files at random, this is not good.
        echo "$ddt: md5sum checks all failed. Volume is possibly unreadable. Review $lgdr/$tdt-$lsnpid-hsh.log. Setting badfile. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    
    if [[ $failcount -gt 1 ]]; then
        # simple notification is in order
        echo "$ddt: md5sum checks failed $failcount times. Since some checks succeeded, it is likely that the failures represent file-permission issues. Please verify by reviewing $lgdr/$tdt-$lsnpid-hsh.log." | tee -a $lgfl
    fi
    
    sleep 10
    echo "$ddt: Backup data verification complete, unmounting volume $crdvolid." | tee -a $lgfl
    

    sync
    sleep 20
    # unmount
    umount /dev/xvd$devdesig
    #check exit code
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi
    
    if [[ $status -gt 0 ]]; then
        echo "$ddt:  umount exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    echo "$ddt:  File system unmounted." | tee -a $lgfl
    sleep 10

    #this will take a LOOOONG time (running on backup target ONCE, PRIOR to backup may make this much faster).
    #e2fsck -fcy /dev/xvdf
    
    #detach volume
    ec2-detach-volume --region $region  $crdvolid --instance $admsrvid -d /dev/sd$devdesig 2>error.txt
    #ATTACHMENT      vol-1cd6687d    i-59386d3f      /dev/sdf        detaching      2012-07-16T23:11:49+0000
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        if [ -s "error.txt" ]; then
            echo -n "$ddt:  " | tee -a $lgfl
            cat error.txt | tee -a $lgfl
        fi
        rm -f error.txt
        echo "$ddt:  detach volume exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    rm -f error.txt

    sleep 10

    #verify volume detach
    dtchstatus=false
    dtchvolstatus=false
    cntr=1
    while [[ "${dtchstatus}" != "available" && "${dtchvolstatus}" != "available" && cntr -lt 5 ]]
    do
        ec2-describe-volumes --region $region  $crdvolid > verifydetach.txt
        cat verifydetach.txt | grep "ATTACHMENT" > dtverif.txt
        cat verifydetach.txt | grep "VOLUME" > dtvolverif.txt
        #VOLUME  vol-e87efa89    500     snap-44831835   us-east-1c      in-use  2012-07-17T20:00:50+0000
        #ATTACHMENT      vol-e87efa89    i-59386d3f      /dev/sdg        attached       2012-07-17T20:02:40+0000
        cat dtverif.txt | awk '{print $5}' > dtchstatus.txt
        cat dtvolverif.txt | awk '{print $6}' > dtchvolstatus.txt
        dtchstatus=`cat dtchstatus.txt`
        dtchvolstatus=`cat dtchvolstatus.txt`
        #rm -f dtverif.txt
        #rm -f dtvolverif.txt
        #rm -f dtchstatus.txt
        #rm -f dtchvolstatus.txt
        #rm -f verifydetach.txt
        # $dtchstatus = when successfully detached, the file should be empty, so var may retain its old value: "detaching", "False", or "attached".
        # $dtchvolstatus = when successfully detached, should be "available". ("in-use" means still-attached).
        echo "$ddt:  describe volumes attach status returned $dtchstatus for $crdvolid on detatch try $cntr." | tee -a $lgfl
        echo "$ddt:  describe volumes volume status returned $dtchvolstatus for $crdvolid on detatch try $cntr." | tee -a $lgfl
        cntr=`expr $cntr + 1`
        sleep 10
    done

    if [ "${dtchstatus}" != "available" ]; then
        #we exited the loop because $cntr exceeded 5
        # (we retry with the --force option. . .  - the backup hasn't failed at this point, amazon's just being stubborn.)
        echo "$ddt:  describe volumes did not return attached after $cntr retries. Forcing a detach." | tee -a $lgfl
        #force if necessary
        ec2-detach-volume --region $region  $crdvolid --instance $admsrvid -d /dev/sd$devdesig --force | tee -a $lgfl
    fi
    sleep 10

    #delete - (no positive detach verify?)
    echo "$ddt:  Deleting volume $crvolid:" | tee -a $lgfl
    ec2-delete-volume --region $region  $crdvolid >> $lgfl 2>error.txt
    #VOLUME  vol-1cd6687d
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        if [ -s "error.txt" ]; then
            echo -n "$ddt:  " | tee -a $lgfl
            cat error.txt | tee -a $lgfl
        fi
        rm -f error.txt
        echo "$ddt:  delete volume exited with '$displaystatus'." | tee -a $lgfl
    fi
    sleep 10

    #verify deletion?
    echo "$ddt:  Verifying deletion of $crvolid with describe volumes:" | tee -a $lgfl
    ec2-describe-volumes --region $region  $crdvolid | tee -a $lgfl
    #Client.InvalidVolume.NotFound: The volume 'vol-1cd6687d' does not exist.
    if [[ $? -gt 0 ]]; then
        echo "$ddt: describe volume returned $?." | tee -a $lgfl
    fi
    sleep 10


    # verify that the snapshot still exists; bail if it's missing.
    ec2-describe-snapshots --region $region  $lsnpid >> $lgfl 2>error.txt
    status=$?
    displaystatus=$status
    if [ -z "$status" ]; then 
        #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
        # I want to capture > 1 situations.
        $status=1
    fi

    if [[ $status -gt 0 ]]; then
        if [ -s "error.txt" ]; then
            echo -n "$ddt:  " | tee -a $lgfl
            cat error.txt | tee -a $lgfl
        fi
        rm -f error.txt
        echo "$ddt:  Describe snapshot exited with '$displaystatus'. Exiting backup." | tee -a $lgfl
        if [ -s "$aspr_home/adminemails.txt" ]; then
            echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
            /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
        fi
        echo "$ddt" > badfile
        exit
    fi
    rm -f error.txt
else
    #this else continues from $novalid test
    echo "$ddt: novalid flag was set; validation was skipped." | tee -a $lgfl
fi

echo "$ddt: Running describe snapshot to determine purgeable snapshots." | tee -a $lgfl
#PURGE OLD SNAPSHOTS
ec2-describe-snapshots --region $region  | grep "$vid" > prvsnps.txt
ec2-describe-snapshots --region $region  2>error.txt
status=$?
displaystatus=$status
if [ -z "$status" ]; then 
    #tests for exit status were bombing if empty and ec2 tools were randomly exiting with ""
    # I want to capture > 1 situations.
    $status=1
fi

if [[ $status -gt 0 ]]; then
    if [ -s "error.txt" ]; then
        echo -n "$ddt:  " | tee -a $lgfl
        cat error.txt | tee -a $lgfl
    fi
    rm -f error.txt
    echo "$ddt:  describe snapshots exited with '$displaystatus'. amisnaps was unable to use the ec2 command line tools. Exiting backup." | tee -a $lgfl
    if [ -s "$aspr_home/adminemails.txt" ]; then
        echo "$ddt:  sending notification e-mail sent to "`/bin/cat $aspr_home/adminemails.txt`"." | tee -a $lgfl
        /usr/bin/mail `/bin/cat $aspr_home/adminemails.txt` -s `/usr/bin/basename $0 `": backup failed on "`/bin/hostname` < $lgfl
    fi
    echo "$ddt" > badfile
    exit
fi
rm -f error.txt


#determine number of previous snapshots.
snpout="$(wc -l prvsnps.txt)"
snpct="$(wc -l prvsnps.txt | awk '{print $1}')"
echo "$ddt: $snpct previous snapshots of this volume were found." | tee -a $lgfl
if [[ $snpct -gt $snaps ]]; then
    # there are purgeable snapshots
    prgable=$(($snpct-$snaps))
    clp=$((prgable+1))
    echo "$ddt: $prgable snapshots are purgeable. " | tee -a $lgfl
    #iterate over list of output
    # -- SORT BY MOST RECENT
    sort prvsnps.txt -k5 -o srtsnps.txt
    # -- remove latest $snaps items from list.
    sed -i $((prgable)),+$((snaps))d srtsnps.txt
    cat srtsnps.txt | awk '{print $2}' > snapidlst.txt
    # - iterate through list:
    while read line
    do
        echo "$ddt: deleting snapshot: $line" | tee -a $lgfl
        ec2-delete-snapshot --region $region  $line
        #remove old file lists and hash logs also
        rm -f $lgdr/????-??-??-$line-hsh.log
        rm -f $lgdr/????-??-??-$line-lst.log
    done < snapidlst.txt
fi

echo "$ddt: amisnaps backup, verification, and purge process completed." | tee -a $lgfl
echo "END**************************************************************" | tee -a $lgfl

